import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.impute import SimpleImputer
from sklearn.preprocessing import StandardScaler, OneHotEncoder
from sklearn.decomposition import PCA
from sklearn.cluster import KMeans, AgglomerativeClustering
from sklearn.metrics import silhouette_score, mean_squared_error, r2_score, make_scorer
from sklearn.ensemble import RandomForestRegressor
from sklearn.linear_model import LinearRegression
from sklearn.model_selection import train_test_split, cross_val_score
from sklearn.tree import DecisionTreeRegressor

# ----------------------------
# 1. 读取数据并合并，标记训练/测试
train = pd.read_csv('train.csv')
test = pd.read_csv('test.csv')

train['is_train'] = 1
test['is_train'] = 0
all_data = pd.concat([train, test], axis=0).reset_index(drop=True)

# ----------------------------
# 2. 缺失值处理
num_cols = all_data.select_dtypes(include=np.number).columns
cat_cols = all_data.select_dtypes(include='object').columns

num_imputer = SimpleImputer(strategy='median')
all_data[num_cols] = num_imputer.fit_transform(all_data[num_cols])

cat_imputer = SimpleImputer(strategy='most_frequent')
all_data[cat_cols] = cat_imputer.fit_transform(all_data[cat_cols])

# ----------------------------
# 3. 异常值检测及剔除
# 设置支持中文字体
plt.rcParams['font.sans-serif'] = ['SimHei']  # 使用黑体
plt.rcParams['axes.unicode_minus'] = False    # 正确显示负号
plt.figure(figsize=(8,4))
sns.boxplot(x=all_data['GrLivArea'])
plt.title('GrLivArea异常值检测')
plt.show()

all_data = all_data[all_data['GrLivArea'] <= 4000].reset_index(drop=True)

# ----------------------------
# 4. 特征工程 - 新增房龄
all_data['HouseAge'] = all_data['YrSold'] - all_data['YearBuilt']

# 重新更新数值型特征列（包含 HouseAge）
num_cols = all_data.select_dtypes(include=np.number).columns

# ----------------------------
# 5. 标准化 & 编码
scaler = StandardScaler()
scaled_num = scaler.fit_transform(all_data[num_cols])

encoder = OneHotEncoder(handle_unknown='ignore')
encoded_cat = encoder.fit_transform(all_data[cat_cols]).toarray()

# 合并为最终特征矩阵
processed_data = pd.concat([
    pd.DataFrame(scaled_num, columns=num_cols),
    pd.DataFrame(encoded_cat, columns=encoder.get_feature_names_out(cat_cols))
], axis=1)

# ----------------------------
# 6. 分离处理后训练和测试集
train_processed = processed_data[all_data['is_train'] == 1].reset_index(drop=True)
test_processed = processed_data[all_data['is_train'] == 0].reset_index(drop=True)

train = all_data[all_data['is_train'] == 1].reset_index(drop=True)
y = train['SalePrice']

# ----------------------------
# 7. 聚类分析部分（PCA降维 + KMeans + 层次聚类 + 可视化）

pca = PCA(n_components=3)
pca_data = pca.fit_transform(train_processed)

# 肘部法则找K
inertia = []
for k in range(2, 10):
    kmeans = KMeans(n_clusters=k, random_state=42)
    kmeans.fit(pca_data)
    inertia.append(kmeans.inertia_)

plt.plot(range(2,10), inertia, marker='o')
plt.xlabel('簇数K')
plt.ylabel('Inertia')
plt.title('肘部法则')
plt.show()  # 选择 K=4

# 层次聚类
agg = AgglomerativeClustering(n_clusters=4, linkage='ward')
agg_labels = agg.fit_predict(pca_data)

# KMeans聚类
kmeans = KMeans(n_clusters=4, random_state=42)
clusters = kmeans.fit_predict(pca_data)

# 3D聚类散点图
fig = plt.figure(figsize=(10,7))
ax = fig.add_subplot(111, projection='3d')
scatter = ax.scatter(pca_data[:,0], pca_data[:,1], pca_data[:,2], c=clusters, cmap='viridis', s=20)
ax.set_xlabel('PCA1')
ax.set_ylabel('PCA2')
ax.set_zlabel('PCA3')
plt.title('K-Means聚类结果（K=4）')
plt.colorbar(scatter)
plt.show()

# 聚类结果特征分析
cluster_profile = train.groupby(clusters).agg({
    'SalePrice': 'mean',
    'GrLivArea': 'median',
    'HouseAge': 'median',
    'Neighborhood': lambda x: x.mode()[0]
})
print(cluster_profile)

# ----------------------------
# 8. 训练/验证集拆分 & 模型训练评估

X_train, X_val, y_train, y_val = train_test_split(train_processed, y, test_size=0.2, random_state=42)

models = {
    'DecisionTree': DecisionTreeRegressor(random_state=42),   # 用决策树替换线性回归
    'RandomForest': RandomForestRegressor(n_estimators=100, random_state=42)
}

results = {}
for name, model in models.items():
    model.fit(X_train, y_train)
    pred = model.predict(X_val)
    results[name] = {
        'RMSE': np.sqrt(mean_squared_error(y_val, pred)),
        'R²': r2_score(y_val, pred)
    }

print(pd.DataFrame(results).T)


models = {
    'DecisionTree': DecisionTreeRegressor(random_state=42),
    'RandomForest': RandomForestRegressor(n_estimators=100, random_state=42)
}

results = {}

# 自定义RMSE评分函数
rmse_scorer = make_scorer(mean_squared_error, squared=False)

for name, model in models.items():
    # 交叉验证计算RMSE（neg_mean_squared_error先转为正，再开根号）
    neg_mse_scores = cross_val_score(model, train_processed, y,
                                     scoring='neg_mean_squared_error', cv=5)
    rmse_scores = np.sqrt(-neg_mse_scores)
    # 交叉验证计算R2
    r2_scores = cross_val_score(model, train_processed, y, scoring='r2', cv=5)

    results[name] = {
        'RMSE Mean': rmse_scores.mean(),
        'RMSE Std': rmse_scores.std(),
        'R² Mean': r2_scores.mean(),
        'R² Std': r2_scores.std()
    }

print(pd.DataFrame(results).T)
